"""Failure Detection and System Health Monitoring.

Provides comprehensive failure detection, system health monitoring,
and automatic rollback trigger mechanisms for system reliability.
"""

import asyncio
import json
import logging
import time
import psutil
import subprocess
from pathlib import Path
from typing import Dict, List, Optional, Callable, Any, Union
from dataclasses import dataclass, asdict
from enum import Enum
from datetime import datetime, timedelta
import hashlib


logger = logging.getLogger(__name__)


class HealthStatus(Enum):
    """System health status enumeration."""
    HEALTHY = "HEALTHY"
    DEGRADED = "DEGRADED"
    CRITICAL = "CRITICAL"
    UNKNOWN = "UNKNOWN"


class FailureType(Enum):
    """Failure type classification."""
    SERVICE_FAILURE = "SERVICE_FAILURE"
    RESOURCE_EXHAUSTION = "RESOURCE_EXHAUSTION"
    CONNECTIVITY_LOSS = "CONNECTIVITY_LOSS"
    FILE_CORRUPTION = "FILE_CORRUPTION"
    DEPENDENCY_MISSING = "DEPENDENCY_MISSING"
    STARTUP_FAILURE = "STARTUP_FAILURE"
    FUNCTIONAL_FAILURE = "FUNCTIONAL_FAILURE"
    USER_INITIATED = "USER_INITIATED"


@dataclass
class ResourceStatus:
    """System resource status information."""
    cpu_percent: float
    memory_percent: float
    disk_usage_percent: float
    disk_free_mb: float
    load_average: List[float]
    temperature: Optional[float] = None
    status: HealthStatus = HealthStatus.HEALTHY
    
    def to_dict(self) -> Dict:
        """Convert to dictionary."""
        return asdict(self)


@dataclass
class ServiceStatus:
    """Service status information."""
    service_name: str
    is_active: bool
    is_enabled: bool
    restart_count: int
    last_restart_time: Optional[datetime]
    status: HealthStatus = HealthStatus.HEALTHY
    
    def to_dict(self) -> Dict:
        """Convert to dictionary."""
        data = asdict(self)
        if self.last_restart_time:
            data['last_restart_time'] = self.last_restart_time.isoformat()
        return data


@dataclass
class ConnectivityStatus:
    """Connectivity status information."""
    network_available: bool
    ble_available: bool
    internet_reachable: bool
    dns_resolution_time_ms: Optional[float]
    connection_failures: int
    status: HealthStatus = HealthStatus.HEALTHY
    
    def to_dict(self) -> Dict:
        """Convert to dictionary."""
        return asdict(self)


@dataclass
class FileIntegrityStatus:
    """File integrity status information."""
    critical_files_intact: bool
    corrupted_files: List[str]
    missing_files: List[str]
    checksum_mismatches: List[str]
    status: HealthStatus = HealthStatus.HEALTHY
    
    def to_dict(self) -> Dict:
        """Convert to dictionary."""
        return asdict(self)


@dataclass
class FailureEvent:
    """Failure event information."""
    timestamp: datetime
    failure_type: FailureType
    severity: HealthStatus
    message: str
    details: Dict[str, Any]
    affected_services: List[str]
    system_state: Optional[Dict] = None
    
    def to_dict(self) -> Dict:
        """Convert to dictionary."""
        data = asdict(self)
        data['timestamp'] = self.timestamp.isoformat()
        data['failure_type'] = self.failure_type.value
        data['severity'] = self.severity.value
        return data


@dataclass
class DiagnosticReport:
    """System diagnostic report."""
    failure_event: FailureEvent
    resource_status: ResourceStatus
    service_status: Dict[str, ServiceStatus]
    connectivity_status: ConnectivityStatus
    file_integrity_status: FileIntegrityStatus
    root_cause_analysis: str
    suggested_actions: List[str]
    rollback_recommendation: str
    estimated_recovery_time: str
    
    def to_dict(self) -> Dict:
        """Convert to dictionary."""
        return {
            'failure_event': self.failure_event.to_dict(),
            'resource_status': self.resource_status.to_dict(),
            'service_status': {k: v.to_dict() for k, v in self.service_status.items()},
            'connectivity_status': self.connectivity_status.to_dict(),
            'file_integrity_status': self.file_integrity_status.to_dict(),
            'root_cause_analysis': self.root_cause_analysis,
            'suggested_actions': self.suggested_actions,
            'rollback_recommendation': self.rollback_recommendation,
            'estimated_recovery_time': self.estimated_recovery_time
        }


class SystemHealthChecker:
    """System health checking utilities."""
    
    @staticmethod
    async def check_services(critical_services: List[str]) -> Dict[str, ServiceStatus]:
        """Check critical system services status."""
        service_statuses = {}
        
        for service in critical_services:
            try:
                # Check if service is active
                result = await asyncio.create_subprocess_exec(
                    'systemctl', 'is-active', service,
                    stdout=asyncio.subprocess.PIPE,
                    stderr=asyncio.subprocess.PIPE
                )
                stdout, _ = await result.communicate()
                is_active = stdout.decode().strip() == 'active'
                
                # Check if service is enabled
                result = await asyncio.create_subprocess_exec(
                    'systemctl', 'is-enabled', service,
                    stdout=asyncio.subprocess.PIPE,
                    stderr=asyncio.subprocess.PIPE
                )
                stdout, _ = await result.communicate()
                is_enabled = stdout.decode().strip() == 'enabled'
                
                # Get restart count (simplified - would use systemctl show in real implementation)
                restart_count = 0
                last_restart_time = None
                
                # Determine status
                if is_active:
                    status = HealthStatus.HEALTHY
                else:
                    status = HealthStatus.CRITICAL
                
                service_statuses[service] = ServiceStatus(
                    service_name=service,
                    is_active=is_active,
                    is_enabled=is_enabled,
                    restart_count=restart_count,
                    last_restart_time=last_restart_time,
                    status=status
                )
                
            except Exception as e:
                logger.error(f"Failed to check service {service}: {e}")
                service_statuses[service] = ServiceStatus(
                    service_name=service,
                    is_active=False,
                    is_enabled=False,
                    restart_count=0,
                    last_restart_time=None,
                    status=HealthStatus.UNKNOWN
                )
        
        return service_statuses
    
    @staticmethod
    def check_resources() -> ResourceStatus:
        """Check system resource usage."""
        try:
            # CPU usage
            cpu_percent = psutil.cpu_percent(interval=1)
            
            # Memory usage
            memory = psutil.virtual_memory()
            memory_percent = memory.percent
            
            # Disk usage
            disk = psutil.disk_usage('/')
            disk_usage_percent = (disk.used / disk.total) * 100
            disk_free_mb = disk.free / (1024 ** 2)
            
            # Load average
            try:
                load_avg = psutil.getloadavg()
                load_average = list(load_avg)
            except (AttributeError, OSError):
                load_average = [0.0, 0.0, 0.0]
            
            # Temperature (if available)
            temperature = None
            try:
                temps = psutil.sensors_temperatures()
                if temps and 'cpu_thermal' in temps:
                    temperature = temps['cpu_thermal'][0].current
            except (AttributeError, OSError):
                pass
            
            # Determine overall status
            status = HealthStatus.HEALTHY
            if cpu_percent > 90 or memory_percent > 85 or disk_usage_percent > 90:
                status = HealthStatus.CRITICAL
            elif cpu_percent > 70 or memory_percent > 70 or disk_usage_percent > 75:
                status = HealthStatus.DEGRADED
            
            return ResourceStatus(
                cpu_percent=cpu_percent,
                memory_percent=memory_percent,
                disk_usage_percent=disk_usage_percent,
                disk_free_mb=disk_free_mb,
                load_average=load_average,
                temperature=temperature,
                status=status
            )
            
        except Exception as e:
            logger.error(f"Failed to check resources: {e}")
            return ResourceStatus(
                cpu_percent=0.0,
                memory_percent=0.0,
                disk_usage_percent=0.0,
                disk_free_mb=0.0,
                load_average=[0.0, 0.0, 0.0],
                status=HealthStatus.UNKNOWN
            )
    
    @staticmethod
    async def check_connectivity() -> ConnectivityStatus:
        """Check network and connectivity status."""
        try:
            network_available = False
            internet_reachable = False
            dns_resolution_time_ms = None
            connection_failures = 0
            
            # Check network interfaces
            interfaces = psutil.net_if_stats()
            for interface, stats in interfaces.items():
                if stats.isup and not interface.startswith('lo'):
                    network_available = True
                    break
            
            # Test internet connectivity with timeout
            if network_available:
                try:
                    start_time = time.time()
                    result = await asyncio.create_subprocess_exec(
                        'ping', '-c', '1', '-W', '5', '8.8.8.8',
                        stdout=asyncio.subprocess.PIPE,
                        stderr=asyncio.subprocess.PIPE
                    )
                    await result.communicate()
                    if result.returncode == 0:
                        internet_reachable = True
                        dns_resolution_time_ms = (time.time() - start_time) * 1000
                    else:
                        connection_failures += 1
                except asyncio.TimeoutError:
                    connection_failures += 1
            
            # BLE availability (simplified check)
            ble_available = True  # Would check actual BLE hardware in real implementation
            
            # Determine status
            status = HealthStatus.HEALTHY
            if not network_available or not internet_reachable:
                status = HealthStatus.CRITICAL
            elif connection_failures > 0:
                status = HealthStatus.DEGRADED
            
            return ConnectivityStatus(
                network_available=network_available,
                ble_available=ble_available,
                internet_reachable=internet_reachable,
                dns_resolution_time_ms=dns_resolution_time_ms,
                connection_failures=connection_failures,
                status=status
            )
            
        except Exception as e:
            logger.error(f"Failed to check connectivity: {e}")
            return ConnectivityStatus(
                network_available=False,
                ble_available=False,
                internet_reachable=False,
                dns_resolution_time_ms=None,
                connection_failures=1,
                status=HealthStatus.UNKNOWN
            )
    
    @staticmethod
    def check_file_integrity(critical_files: List[str], 
                           checksums: Optional[Dict[str, str]] = None) -> FileIntegrityStatus:
        """Check critical file integrity."""
        try:
            corrupted_files = []
            missing_files = []
            checksum_mismatches = []
            
            for file_path in critical_files:
                path = Path(file_path)
                
                if not path.exists():
                    missing_files.append(file_path)
                    continue
                
                # Basic file accessibility check
                try:
                    with open(path, 'rb') as f:
                        # Try to read first few bytes
                        f.read(1024)
                except (PermissionError, OSError):
                    corrupted_files.append(file_path)
                    continue
                
                # Checksum verification if provided
                if checksums and file_path in checksums:
                    try:
                        actual_checksum = SystemHealthChecker._calculate_file_hash(path)
                        expected_checksum = checksums[file_path]
                        
                        if actual_checksum != expected_checksum:
                            checksum_mismatches.append(file_path)
                    except Exception:
                        corrupted_files.append(file_path)
            
            # Determine status
            critical_files_intact = len(missing_files) == 0 and len(corrupted_files) == 0
            status = HealthStatus.HEALTHY
            
            if missing_files or corrupted_files:
                status = HealthStatus.CRITICAL
            elif checksum_mismatches:
                status = HealthStatus.DEGRADED
            
            return FileIntegrityStatus(
                critical_files_intact=critical_files_intact,
                corrupted_files=corrupted_files,
                missing_files=missing_files,
                checksum_mismatches=checksum_mismatches,
                status=status
            )
            
        except Exception as e:
            logger.error(f"Failed to check file integrity: {e}")
            return FileIntegrityStatus(
                critical_files_intact=False,
                corrupted_files=[],
                missing_files=[],
                checksum_mismatches=[],
                status=HealthStatus.UNKNOWN
            )
    
    @staticmethod
    def _calculate_file_hash(file_path: Path) -> str:
        """Calculate SHA256 hash of file."""
        hasher = hashlib.sha256()
        with open(file_path, 'rb') as f:
            for chunk in iter(lambda: f.read(8192), b""):
                hasher.update(chunk)
        return hasher.hexdigest()


class FailureDetector:
    """System failure detection and monitoring service."""
    
    def __init__(self, config_path: Optional[str] = None):
        """Initialize failure detector.
        
        Args:
            config_path: Path to detection configuration file
        """
        self.config = self._load_config(config_path)
        self.health_checker = SystemHealthChecker()
        
        # Detection state
        self.is_monitoring = False
        self.failure_history: List[FailureEvent] = []
        self.last_health_check: Optional[datetime] = None
        
        # Callbacks
        self.failure_callback: Optional[Callable] = None
        self.health_callback: Optional[Callable] = None
        
        # Monitoring task
        self._monitor_task: Optional[asyncio.Task] = None
        
        # Service restart tracking
        self._service_restart_counts: Dict[str, int] = {}
        self._service_last_check: Dict[str, datetime] = {}
    
    def _load_config(self, config_path: Optional[str]) -> Dict:
        """Load failure detection configuration."""
        default_config = {
            "service_failures": {
                "max_restart_attempts": 3,
                "restart_interval_seconds": 10,
                "critical_services": [
                    "cloud-printer-ble.service",
                    "cloud-printer-network.service"
                ]
            },
            "resource_monitoring": {
                "max_cpu_percent": 90,
                "max_memory_percent": 85,
                "min_disk_space_mb": 100,
                "monitoring_interval_seconds": 30
            },
            "connectivity": {
                "network_timeout_seconds": 30,
                "max_connection_failures": 5,
                "ble_connection_timeout": 15,
                "check_interval_seconds": 60
            },
            "file_integrity": {
                "critical_files": [
                    "/usr/local/bin/cloud-printer",
                    "/usr/local/lib/libprinter.so"
                ],
                "check_interval_seconds": 300,
                "checksums": {}
            },
            "general": {
                "health_check_interval_seconds": 30,
                "max_failure_history": 100,
                "auto_rollback_enabled": True,
                "rollback_cooldown_seconds": 300
            }
        }
        
        if config_path and Path(config_path).exists():
            try:
                with open(config_path, 'r') as f:
                    user_config = json.load(f)
                # Merge with default config
                default_config.update(user_config)
            except Exception as e:
                logger.warning(f"Failed to load config from {config_path}: {e}")
        
        return default_config
    
    def set_failure_callback(self, callback: Callable[[FailureEvent, DiagnosticReport], None]):
        """Set callback for failure events.
        
        Args:
            callback: Function to call when failure is detected
        """
        self.failure_callback = callback
    
    def set_health_callback(self, callback: Callable[[Dict[str, Any]], None]):
        """Set callback for health status updates.
        
        Args:
            callback: Function to call with health status
        """
        self.health_callback = callback
    
    async def start_monitoring(self):
        """Start continuous system monitoring."""
        if self.is_monitoring:
            logger.warning("Monitoring already active")
            return
        
        self.is_monitoring = True
        self._monitor_task = asyncio.create_task(self._monitoring_loop())
        logger.info("Failure detector monitoring started")
    
    async def stop_monitoring(self):
        """Stop system monitoring."""
        self.is_monitoring = False
        if self._monitor_task:
            self._monitor_task.cancel()
            try:
                await self._monitor_task
            except asyncio.CancelledError:
                pass
        logger.info("Failure detector monitoring stopped")
    
    async def _monitoring_loop(self):
        """Main monitoring loop."""
        try:
            interval = self.config['general']['health_check_interval_seconds']
            
            while self.is_monitoring:
                try:
                    # Perform comprehensive health check
                    await self._perform_health_check()
                    
                    # Wait for next check
                    await asyncio.sleep(interval)
                    
                except Exception as e:
                    logger.error(f"Error in monitoring loop: {e}")
                    await asyncio.sleep(interval)  # Continue monitoring despite errors
                    
        except asyncio.CancelledError:
            logger.info("Monitoring loop cancelled")
    
    async def _perform_health_check(self):
        """Perform comprehensive system health check."""
        try:
            self.last_health_check = datetime.now()
            
            # Check all system components
            resource_status = self.health_checker.check_resources()
            service_status = await self.health_checker.check_services(
                self.config['service_failures']['critical_services']
            )
            connectivity_status = await self.health_checker.check_connectivity()
            file_integrity_status = self.health_checker.check_file_integrity(
                self.config['file_integrity']['critical_files'],
                self.config['file_integrity']['checksums']
            )
            
            # Analyze for failures
            failures = self._analyze_health_status(
                resource_status, service_status, 
                connectivity_status, file_integrity_status
            )
            
            # Process detected failures
            for failure in failures:
                await self._handle_failure(
                    failure, resource_status, service_status,
                    connectivity_status, file_integrity_status
                )
            
            # Send health status update
            if self.health_callback:
                health_data = {
                    'timestamp': self.last_health_check.isoformat(),
                    'overall_status': self._calculate_overall_health(
                        resource_status, service_status,
                        connectivity_status, file_integrity_status
                    ).value,
                    'resource_status': resource_status.to_dict(),
                    'service_status': {k: v.to_dict() for k, v in service_status.items()},
                    'connectivity_status': connectivity_status.to_dict(),
                    'file_integrity_status': file_integrity_status.to_dict()
                }
                
                try:
                    await self.health_callback(health_data)
                except Exception as e:
                    logger.error(f"Error in health callback: {e}")
            
        except Exception as e:
            logger.error(f"Error performing health check: {e}")
    
    def _analyze_health_status(self, resource_status: ResourceStatus,
                              service_status: Dict[str, ServiceStatus],
                              connectivity_status: ConnectivityStatus,
                              file_integrity_status: FileIntegrityStatus) -> List[FailureEvent]:
        """Analyze health status and detect failures."""
        failures = []
        current_time = datetime.now()
        
        # Check resource failures
        if resource_status.status == HealthStatus.CRITICAL:
            if resource_status.cpu_percent > self.config['resource_monitoring']['max_cpu_percent']:
                failures.append(FailureEvent(
                    timestamp=current_time,
                    failure_type=FailureType.RESOURCE_EXHAUSTION,
                    severity=HealthStatus.CRITICAL,
                    message=f"CPU usage critical: {resource_status.cpu_percent:.1f}%",
                    details={'cpu_percent': resource_status.cpu_percent},
                    affected_services=[]
                ))
            
            if resource_status.memory_percent > self.config['resource_monitoring']['max_memory_percent']:
                failures.append(FailureEvent(
                    timestamp=current_time,
                    failure_type=FailureType.RESOURCE_EXHAUSTION,
                    severity=HealthStatus.CRITICAL,
                    message=f"Memory usage critical: {resource_status.memory_percent:.1f}%",
                    details={'memory_percent': resource_status.memory_percent},
                    affected_services=[]
                ))
            
            if resource_status.disk_free_mb < self.config['resource_monitoring']['min_disk_space_mb']:
                failures.append(FailureEvent(
                    timestamp=current_time,
                    failure_type=FailureType.RESOURCE_EXHAUSTION,
                    severity=HealthStatus.CRITICAL,
                    message=f"Disk space critical: {resource_status.disk_free_mb:.0f}MB free",
                    details={'disk_free_mb': resource_status.disk_free_mb},
                    affected_services=[]
                ))
        
        # Check service failures
        failed_services = []
        for service_name, status in service_status.items():
            if status.status == HealthStatus.CRITICAL:
                failed_services.append(service_name)
        
        if failed_services:
            failures.append(FailureEvent(
                timestamp=current_time,
                failure_type=FailureType.SERVICE_FAILURE,
                severity=HealthStatus.CRITICAL,
                message=f"Critical services failed: {', '.join(failed_services)}",
                details={'failed_services': failed_services},
                affected_services=failed_services
            ))
        
        # Check connectivity failures
        if connectivity_status.status == HealthStatus.CRITICAL:
            failures.append(FailureEvent(
                timestamp=current_time,
                failure_type=FailureType.CONNECTIVITY_LOSS,
                severity=HealthStatus.CRITICAL,
                message="Network connectivity lost",
                details={
                    'network_available': connectivity_status.network_available,
                    'internet_reachable': connectivity_status.internet_reachable
                },
                affected_services=[]
            ))
        
        # Check file integrity failures
        if file_integrity_status.status == HealthStatus.CRITICAL:
            failures.append(FailureEvent(
                timestamp=current_time,
                failure_type=FailureType.FILE_CORRUPTION,
                severity=HealthStatus.CRITICAL,
                message=f"Critical files corrupted or missing",
                details={
                    'corrupted_files': file_integrity_status.corrupted_files,
                    'missing_files': file_integrity_status.missing_files
                },
                affected_services=[]
            ))
        
        return failures
    
    def _calculate_overall_health(self, resource_status: ResourceStatus,
                                 service_status: Dict[str, ServiceStatus],
                                 connectivity_status: ConnectivityStatus,
                                 file_integrity_status: FileIntegrityStatus) -> HealthStatus:
        """Calculate overall system health status."""
        statuses = [
            resource_status.status,
            connectivity_status.status,
            file_integrity_status.status
        ]
        
        # Add service statuses
        for service in service_status.values():
            statuses.append(service.status)
        
        # Determine overall status (worst case)
        if HealthStatus.CRITICAL in statuses:
            return HealthStatus.CRITICAL
        elif HealthStatus.DEGRADED in statuses:
            return HealthStatus.DEGRADED
        elif HealthStatus.UNKNOWN in statuses:
            return HealthStatus.UNKNOWN
        else:
            return HealthStatus.HEALTHY
    
    async def _handle_failure(self, failure: FailureEvent,
                             resource_status: ResourceStatus,
                             service_status: Dict[str, ServiceStatus],
                             connectivity_status: ConnectivityStatus,
                             file_integrity_status: FileIntegrityStatus):
        """Handle detected failure event."""
        # Add to history
        self.failure_history.append(failure)
        
        # Limit history size
        max_history = self.config['general']['max_failure_history']
        if len(self.failure_history) > max_history:
            self.failure_history = self.failure_history[-max_history:]
        
        logger.error(f"Failure detected: {failure.message}")
        
        # Generate diagnostic report
        diagnostic_report = self._generate_diagnostic_report(
            failure, resource_status, service_status,
            connectivity_status, file_integrity_status
        )
        
        # Call failure callback
        if self.failure_callback:
            try:
                await self.failure_callback(failure, diagnostic_report)
            except Exception as e:
                logger.error(f"Error in failure callback: {e}")
    
    def _generate_diagnostic_report(self, failure: FailureEvent,
                                   resource_status: ResourceStatus,
                                   service_status: Dict[str, ServiceStatus],
                                   connectivity_status: ConnectivityStatus,
                                   file_integrity_status: FileIntegrityStatus) -> DiagnosticReport:
        """Generate comprehensive diagnostic report."""
        # Root cause analysis
        root_cause = self._analyze_root_cause(failure, resource_status, service_status)
        
        # Suggested actions
        suggested_actions = self._generate_suggested_actions(failure, root_cause)
        
        # Rollback recommendation
        rollback_recommendation = self._assess_rollback_need(failure, root_cause)
        
        # Estimated recovery time
        estimated_recovery_time = self._estimate_recovery_time(failure, rollback_recommendation)
        
        return DiagnosticReport(
            failure_event=failure,
            resource_status=resource_status,
            service_status=service_status,
            connectivity_status=connectivity_status,
            file_integrity_status=file_integrity_status,
            root_cause_analysis=root_cause,
            suggested_actions=suggested_actions,
            rollback_recommendation=rollback_recommendation,
            estimated_recovery_time=estimated_recovery_time
        )
    
    def _analyze_root_cause(self, failure: FailureEvent,
                           resource_status: ResourceStatus,
                           service_status: Dict[str, ServiceStatus]) -> str:
        """Analyze root cause of failure."""
        if failure.failure_type == FailureType.SERVICE_FAILURE:
            # Check for resource constraints affecting services
            if resource_status.memory_percent > 80:
                return "Service failure likely due to memory exhaustion"
            elif resource_status.cpu_percent > 85:
                return "Service failure likely due to CPU overload"
            elif resource_status.disk_free_mb < 50:
                return "Service failure likely due to insufficient disk space"
            else:
                return "Service failure due to configuration or dependency issues"
        
        elif failure.failure_type == FailureType.RESOURCE_EXHAUSTION:
            return "System resource exhaustion detected"
        
        elif failure.failure_type == FailureType.CONNECTIVITY_LOSS:
            return "Network connectivity issues detected"
        
        elif failure.failure_type == FailureType.FILE_CORRUPTION:
            return "Critical system files corrupted or missing"
        
        else:
            return "Unknown failure cause - requires manual investigation"
    
    def _generate_suggested_actions(self, failure: FailureEvent, root_cause: str) -> List[str]:
        """Generate suggested recovery actions."""
        actions = []
        
        if failure.failure_type == FailureType.SERVICE_FAILURE:
            actions.extend([
                "Restart failed services",
                "Check service logs for errors",
                "Verify service dependencies",
                "Check system resources"
            ])
        
        elif failure.failure_type == FailureType.RESOURCE_EXHAUSTION:
            actions.extend([
                "Free up system resources",
                "Kill non-essential processes",
                "Clean temporary files",
                "Check for memory leaks"
            ])
        
        elif failure.failure_type == FailureType.CONNECTIVITY_LOSS:
            actions.extend([
                "Check network interface status",
                "Verify network configuration",
                "Test network connectivity",
                "Restart network services"
            ])
        
        elif failure.failure_type == FailureType.FILE_CORRUPTION:
            actions.extend([
                "Verify file system integrity",
                "Restore files from backup",
                "Check disk health",
                "Consider system rollback"
            ])
        
        # Add rollback suggestion for critical failures
        if failure.severity == HealthStatus.CRITICAL:
            actions.append("Consider rollback to previous stable version")
        
        return actions
    
    def _assess_rollback_need(self, failure: FailureEvent, root_cause: str) -> str:
        """Assess whether rollback is recommended."""
        if not self.config['general']['auto_rollback_enabled']:
            return "manual"
        
        critical_failures = [
            FailureType.SERVICE_FAILURE,
            FailureType.FILE_CORRUPTION,
            FailureType.STARTUP_FAILURE
        ]
        
        if failure.failure_type in critical_failures and failure.severity == HealthStatus.CRITICAL:
            return "automatic"
        elif failure.severity == HealthStatus.CRITICAL:
            return "recommended"
        else:
            return "not_recommended"
    
    def _estimate_recovery_time(self, failure: FailureEvent, rollback_recommendation: str) -> str:
        """Estimate recovery time based on failure type and recommended action."""
        if rollback_recommendation == "automatic":
            return "00:03:00"  # 3 minutes for automatic rollback
        elif rollback_recommendation == "recommended":
            return "00:05:00"  # 5 minutes for manual rollback
        
        # Estimate based on failure type
        time_estimates = {
            FailureType.SERVICE_FAILURE: "00:01:00",
            FailureType.RESOURCE_EXHAUSTION: "00:02:00",
            FailureType.CONNECTIVITY_LOSS: "00:01:30",
            FailureType.FILE_CORRUPTION: "00:04:00",
            FailureType.STARTUP_FAILURE: "00:03:00"
        }
        
        return time_estimates.get(failure.failure_type, "00:05:00")
    
    async def manual_health_check(self) -> Dict[str, Any]:
        """Perform manual health check and return results."""
        resource_status = self.health_checker.check_resources()
        service_status = await self.health_checker.check_services(
            self.config['service_failures']['critical_services']
        )
        connectivity_status = await self.health_checker.check_connectivity()
        file_integrity_status = self.health_checker.check_file_integrity(
            self.config['file_integrity']['critical_files'],
            self.config['file_integrity']['checksums']
        )
        
        overall_status = self._calculate_overall_health(
            resource_status, service_status,
            connectivity_status, file_integrity_status
        )
        
        return {
            'timestamp': datetime.now().isoformat(),
            'overall_status': overall_status.value,
            'resource_status': resource_status.to_dict(),
            'service_status': {k: v.to_dict() for k, v in service_status.items()},
            'connectivity_status': connectivity_status.to_dict(),
            'file_integrity_status': file_integrity_status.to_dict()
        }
    
    def get_failure_history(self, limit: Optional[int] = None) -> List[Dict]:
        """Get failure history."""
        history = self.failure_history
        if limit:
            history = history[-limit:]
        
        return [failure.to_dict() for failure in history]
    
    def is_system_healthy(self) -> bool:
        """Check if system is currently healthy."""
        if not self.last_health_check:
            return False
        
        # Check if health check is recent
        time_since_check = datetime.now() - self.last_health_check
        if time_since_check.total_seconds() > 60:  # More than 1 minute ago
            return False
        
        # Check recent failures
        recent_failures = [
            f for f in self.failure_history
            if (datetime.now() - f.timestamp).total_seconds() < 300  # Last 5 minutes
        ]
        
        return len(recent_failures) == 0


# Example usage for testing
if __name__ == "__main__":
    import asyncio
    
    logging.basicConfig(level=logging.INFO)
    
    async def test_failure_detector():
        """Test failure detector functionality."""
        detector = FailureDetector()
        
        # Set up callbacks
        async def on_failure(failure_event, diagnostic_report):
            print(f"FAILURE DETECTED: {failure_event.message}")
            print(f"Root cause: {diagnostic_report.root_cause_analysis}")
            print(f"Recommended action: {diagnostic_report.rollback_recommendation}")
        
        async def on_health_update(health_data):
            print(f"Health update: {health_data['overall_status']}")
        
        detector.set_failure_callback(on_failure)
        detector.set_health_callback(on_health_update)
        
        # Perform manual health check
        health_status = await detector.manual_health_check()
        print(f"Manual health check: {health_status}")
        
        # Start monitoring for a short period
        await detector.start_monitoring()
        await asyncio.sleep(10)
        await detector.stop_monitoring()
    
    # Run test
    asyncio.run(test_failure_detector())